# -*- coding: utf-8 -*-
"""
Created on Thu Sep  8 14:52:59 2022

@author: 123
"""


# D:\kaggle论文准备\data
import pandas as pd
import numpy as np
df=pd.read_csv(r'D:\kaggle论文准备\data\Training Data.csv')


#更新STATE列的值
df['STATE'] = df['STATE'].map({'Gujarat':'high',
'Haryana':'high',
'Himachal_Pradesh':'high',
'Karnataka':'high',
'Kerala':'high',
'Maharashtra':'high',
'Mizoram':'high',
'Puducherry':'high',
'Tamil_Nadu':'high',
'Telangana':'high',
'Uttarakhand':'high',
'Andhra_Pradesh':'middle',
'Odisha':'middle',
'Punjab':'middle',
'Tripura':'middle',
'Assam':'low',
'Bihar':'low',
'Chhattisgarh':'low',
'Jammu_and_Kashmir':'low',
'Jharkhand':'low',
'Madhya_Pradesh':'low',
'Manipur':'low',
'Rajasthan':'low',
'Uttar_Pradesh':'low',
'Uttar_Pradesh[5]':'low',
'West_Bengal':'low',
'Chandigarh':'Ultra_high',
'Delhi':'Ultra_high',
'Sikkim':'Ultra_high',
np.nan:'NY'},
na_action=None)


#更新Profession列的值
df['Profession'] = df['Profession'].map({
'Air_traffic_controller':'is_business service personnel',
'Analyst':'is_professiona_skill_personnel',
'Architect':'is_professiona_skill_personnel',
'Army_officer':'is_person_in_charge_of_enterprises_and_institutions',
'Artist':'is_professiona_skill_personnel',
'Aviator':'is_professiona_skill_personnel',
'Biomedical_Engineer':'is_professiona_skill_personnel',
'Chartered_Accountant':'is_professiona_skill_personnel',
'Chef':'is_business service personnel',
'Chemical_engineer':'is_professiona_skill_personnel',
'Civil_engineer':'is_professiona_skill_personnel',
'Civil_servant':'is_person_in_charge_of_enterprises_and_institutions',
'Comedian':'is_professiona_skill_personnel',
'Computer_hardware_engineer':'is_professiona_skill_personnel',
'Computer_operator':'is_production_transport_worker',
'Consultant':'is_professiona_skill_personnel',
'Dentist':'is_professiona_skill_personnel',
'Design_Engineer':'is_professiona_skill_personnel',
'Designer':'is_professiona_skill_personnel',
'Drafter':'is_professiona_skill_personnel',
'Economist':'is_professiona_skill_personnel',
'Engineer':'is_professiona_skill_personnel',
'Fashion_Designer':'is_professiona_skill_personnel',
'Financial_Analyst':'is_professiona_skill_personnel',
'Firefighter':'is_handle_affairs_personnel',
'Flight_attendant':'is_business service personnel',
'Geologist':'is_professiona_skill_personnel',
'Graphic_Designer':'is_professiona_skill_personnel',
'Hotel_Manager':'is_person_in_charge_of_enterprises_and_institutions',
'Industrial_Engineer':'is_professiona_skill_personnel',
'Lawyer':'is_professiona_skill_personnel',
'Librarian':'is_professiona_skill_personnel',
'Magistrate':'is_person_in_charge_of_enterprises_and_institutions',
'Mechanical_engineer':'is_professiona_skill_personnel',
'Microbiologist':'is_professiona_skill_personnel',
'Official':'is_person_in_charge_of_enterprises_and_institutions',
'Petroleum_Engineer':'is_professiona_skill_personnel',
'Physician':'is_professiona_skill_personnel',
'Police_officer':'is_person_in_charge_of_enterprises_and_institutions',
'Politician':'is_person_in_charge_of_enterprises_and_institutions',
'Psychologist':'is_professiona_skill_personnel',
'Scientist':'is_professiona_skill_personnel',
'Secretary':'is_handle_affairs_personnel',
'Software_Developer':'is_professiona_skill_personnel',
'Statistician':'is_professiona_skill_personnel',
'Surgeon':'is_professiona_skill_personnel',
'Surveyor':'is_professiona_skill_personnel',
'Technical_writer':'is_professiona_skill_personnel',
'Technician':'is_professiona_skill_personnel',
'Technology_specialist':'is_professiona_skill_personnel',
'Web_designer':'is_professiona_skill_personnel',
np.nan:'NY'},
na_action=None)

grouped_Income=df['Income'].groupby(df['Income'])
grouped_Age=df['Age'].groupby(df['Age'])
grouped_Experience=df['Experience'].groupby(df['Experience'])
#grouped_MarriedSingle=df['Married\\/Single'].groupby(df['Married\\/Single'])
grouped_MarriedSingle=df['Married/Single'].groupby(df['Married/Single'])

grouped_House_Ownership=df['House_Ownership'].groupby(df['House_Ownership'])
grouped_Car_Ownership=df['Car_Ownership'].groupby(df['Car_Ownership'])
grouped_Profession=df['Profession'].groupby(df['Profession'])
grouped_CITY=df['CITY'].groupby(df['CITY'])
grouped_STATE=df['STATE'].groupby(df['STATE'])
grouped_CURRENT_JOB_YRS=df['CURRENT_JOB_YRS'].groupby(df['CURRENT_JOB_YRS'])
grouped_CURRENT_HOUSE_YRS=df['CURRENT_HOUSE_YRS'].groupby(df['CURRENT_HOUSE_YRS'])
grouped_Risk_Flag=df['Risk_Flag'].groupby(df['Risk_Flag'])


#检查一下各列数据分布情况
print(grouped_Income.count())
print(grouped_Age.count())
print(grouped_Experience.count())
print(grouped_MarriedSingle.count())

#实现哑变量编码
dummy_Married = pd.get_dummies(df['Married/Single'], prefix='Married/Single')
dummy_Married.columns

dummy_House_Ownership = pd.get_dummies(df['House_Ownership'], prefix='House_Ownership')
dummy_House_Ownership.columns

dummy_Car_Ownership = pd.get_dummies(df['Car_Ownership'], prefix='Car_Ownership')
dummy_Car_Ownership.columns

dummy_Profession = pd.get_dummies(df['Profession'], prefix='Profession')
dummy_Profession.columns

dummy_STATE = pd.get_dummies(df['STATE'], prefix='STATE')
dummy_STATE.columns



# 数值变量
cols_to_keep=["Risk_Flag","Income","Age","Experience","CURRENT_JOB_YRS","CURRENT_HOUSE_YRS"]

new_df=df[cols_to_keep].join(dummy_Married.loc[:,'Married/Single_single':]).join(dummy_House_Ownership.loc[:,'House_Ownership_owned':]).join(dummy_Car_Ownership.loc[:,'Car_Ownership_yes':]).join(dummy_Profession.loc[:,'Profession_is_handle_affairs_personnel':]).join(dummy_STATE.loc[:,'STATE_high':])

x=new_df.loc["Income":]


#数据标准化处理 

from sklearn.preprocessing import StandardScaler
new_df['Income'] = StandardScaler().fit_transform(new_df[['Income']])
new_df['Age'] = StandardScaler().fit_transform(new_df[['Age']])
new_df['Experience'] = StandardScaler().fit_transform(new_df[['Experience']])
new_df['CURRENT_JOB_YRS'] = StandardScaler().fit_transform(new_df[['CURRENT_JOB_YRS']])
new_df['CURRENT_HOUSE_YRS'] = StandardScaler().fit_transform(new_df[['CURRENT_HOUSE_YRS']])


# 查看样本比例
import numpy as np
import matplotlib.pyplot as plt
num_nonfraud = np.sum(new_df['Risk_Flag'] == 1) #30996
num_fraud = np.sum(new_df['Risk_Flag'] == 0)    #221004


#讲第一步运行数据保存到文件中去
new_df.to_csv("01_step_result.csv")


